{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第5步：调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath + \"RentListingInquries_FE_train.csv\")\n",
    "test = pd.read_csv(dpath + \"RentListingInquries_FE_test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据探索略过,将训练数据集和类别标签进行分割"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis =1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "准备进行交叉验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-8-48403b68634e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-8-48403b68634e>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    第二轮参数调整得到的n_estimators最优值（193），max_depth最优值（6）， min_child_weight最优值（6），上一步中得到subsample最优值（0.7），colsample_bytree最优值（0.7），其余参数继续默认值的情况下调整reg_alpha 和reg_lambda\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "第二轮参数调整得到的n_estimators最优值（193），max_depth最优值（6）， min_child_weight最优值（6），上一步中得到subsample最优值（0.7），colsample_bytree最优值（0.7），其余参数继续默认值的情况下调整reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=193,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.7,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58273, std: 0.00329, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58284, std: 0.00322, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58270, std: 0.00354, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58270, std: 0.00380, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58250, std: 0.00405, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58343, std: 0.00418, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 1},\n",
       " -0.5825017158295256)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/cuiyue/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([268.80008202, 264.1507072 , 264.74773803, 263.05696602,\n",
       "        263.4942378 , 224.00589099]),\n",
       " 'mean_score_time': array([1.00641723, 0.97333961, 1.0397078 , 0.97353182, 0.80999804,\n",
       "        0.73188577]),\n",
       " 'mean_test_score': array([-0.58272899, -0.58284337, -0.58269653, -0.58270064, -0.58250172,\n",
       "        -0.58342883]),\n",
       " 'mean_train_score': array([-0.48758702, -0.48873143, -0.49112745, -0.48956529, -0.49084948,\n",
       "        -0.49329517]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([4, 5, 2, 3, 1, 6], dtype=int32),\n",
       " 'split0_test_score': array([-0.57743633, -0.57707254, -0.57649225, -0.57619413, -0.57527893,\n",
       "        -0.57627457]),\n",
       " 'split0_train_score': array([-0.4891152 , -0.48987547, -0.4927874 , -0.49131349, -0.49162959,\n",
       "        -0.49414583]),\n",
       " 'split1_test_score': array([-0.58071126, -0.58200071, -0.58173382, -0.58092607, -0.58115995,\n",
       "        -0.58172922]),\n",
       " 'split1_train_score': array([-0.4869162 , -0.48871402, -0.49021248, -0.48878495, -0.49054676,\n",
       "        -0.49337276]),\n",
       " 'split2_test_score': array([-0.58368901, -0.58371606, -0.58330932, -0.5847079 , -0.58411831,\n",
       "        -0.58477735]),\n",
       " 'split2_train_score': array([-0.48849937, -0.48880942, -0.49180851, -0.49030543, -0.49130326,\n",
       "        -0.49420162]),\n",
       " 'split3_test_score': array([-0.58521563, -0.58511698, -0.58515282, -0.58459882, -0.58512103,\n",
       "        -0.58592251]),\n",
       " 'split3_train_score': array([-0.48678742, -0.48816828, -0.49051829, -0.48852147, -0.48991169,\n",
       "        -0.49182614]),\n",
       " 'split4_test_score': array([-0.58659389, -0.58631161, -0.58679568, -0.58707762, -0.58683168,\n",
       "        -0.58844201]),\n",
       " 'split4_train_score': array([-0.48661693, -0.48808995, -0.49031056, -0.4889011 , -0.49085612,\n",
       "        -0.49292948]),\n",
       " 'std_fit_time': array([ 2.19587872,  0.85655322,  0.41929927,  0.72740903,  0.51912961,\n",
       "        48.0501876 ]),\n",
       " 'std_score_time': array([0.10741305, 0.18111588, 0.15371827, 0.15086104, 0.10161057,\n",
       "        0.16402985]),\n",
       " 'std_test_score': array([0.00329013, 0.00322403, 0.00353922, 0.0038019 , 0.00405441,\n",
       "        0.00417705]),\n",
       " 'std_train_score': array([0.00101962, 0.00063935, 0.00100944, 0.00107178, 0.0005976 ,\n",
       "        0.0008767 ])}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.582502 using {'reg_alpha': 2, 'reg_lambda': 1}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPXV+PHPyb6xhYQ1QMIiiIAIARGsitUKqOCCC26AivZRH5c+rdVujz9qq7Wtj7VqLUgVXEDFhU20LrgWhLCvsoQIAZSQsAUIZDm/P743MQmBDJDJnSTn/XrNKzN37r1zhjA5c773u4iqYowxxtS0ML8DMMYYUz9ZgjHGGBMUlmCMMcYEhSUYY4wxQWEJxhhjTFBYgjHGGBMUlmCMMcYEhSUYY4wxQWEJxhhjTFBE+B2An5KSkjQ1NdXvMIwxpk5ZvHjxLlVNrm6/Bp1gUlNTycjI8DsMY4ypU0Tk20D2syYyY4wxQWEJxhhjTFBYgjHGGBMUDfoaTFUKCwvJzs6moKDA71BMFWJiYkhJSSEyMtLvUIwx1bAEU0l2djaNGjUiNTUVEfE7HFOOqpKbm0t2djZpaWl+h2OMqYY1kVVSUFBA8+bNLbmEIBGhefPmVl0aU0dYgqmCJZfQZb8bY+oOSzDGGNOQqMJ//g4H84L+UnYNxhhjGoqSEpj7C1j0AiAw8J6gvpxVMA3cp59+ymWXXXbK+xzPunXrOOecc4iOjuYvf/nLMfcbM2YMaWlp9O7dm969e7Ns2bKTfk1jTCUlJTD7PpdcBt0H59wd9Je0CibEqSqqSlhY3f0ukJiYyNNPP827775b7b5//vOfGTlyZC1EZUwDUlIMM+6G5VPhvF/A4F9DLVzPDGqCEZEhwN+AcOAFVX280vNjgD8D27xNz6jqC95zTwCX4qqsD4H7VFVF5H2gtRf7F8Ddqlpc7pw/986ZrKq7TiX+/zdrNWu27zuVUxyle5vG/O/lZxx3n6ysLIYOHcrgwYOZP38+999/P88//zyHDx+mU6dOvPjiiyQkJPDee+/xs5/9jKSkJPr06UNmZiazZ8+u8pwLFy7k/vvv59ChQ8TGxvLiiy/StWvXCvs88sgjbNq0iW3btrF161YefPBBxo0bB0B+fj4jR45k1apV9O3bl1deeQURYfz48cyaNYtDhw4xcOBA/vnPfx51Ib5Fixa0aNGCOXPmnMK/nDHmpBQXwjt3wqq3XGI5/8Fae+mgfS0WkXDgWWAo0B0YJSLdq9j1dVXt7d1Kk8tAYBDQC+gB9APO9/a/VlXP9LYnA9eUe812wMXAluC8q9rzzTffcMstt/Dhhx8yadIkPvroI5YsWUJ6ejpPPvkkBQUF3HnnncydO5cvv/ySnJyc456vW7dufP755yxdupTx48fzq1/9qsr9VqxYwZw5c5g/fz7jx49n+/btACxdupSnnnqKNWvWkJmZyVdffQXAPffcw6JFi1i1ahWHDh0qS3DPP/88zz///Am/71//+tf06tWLBx54gMOHD5/w8caYcoqOwPRbXXK56P/VanKB4FYw/YGNqpoJICLTgBHAmgCOVSAGiAIEiAS+B1DV0pIiwnteyx33f8CDwIwaiL/aSiOYOnTowIABA5g9ezZr1qxh0KBBABw5coRzzjmHdevW0bFjx7IBh6NGjWLChAnHPN/evXsZPXo0GzZsQEQoLCyscr8RI0YQGxtLbGwsgwcPZuHChTRt2pT+/fuTkpICQO/evcnKyuLcc89l3rx5PPHEExw8eJC8vDzOOOMMLr/8cn7605+e8Ht+7LHHaNWqFUeOHOGOO+7gT3/6E7/73e9O+DzGGKDoMLwxGtbPhUseg3PuqvUQgtmw3xbYWu5xtretsqtFZIWITPcqEFR1PjAP2OHdPlDVtaUHiMgHwE5gPzDd2zYc2Kaqy48XlIjcISIZIpJR3bd+P8XHxwPuGszFF1/MsmXLWLZsGWvWrGHSpEmoajVnqOi3v/0tgwcPZtWqVcyaNeuYgxUrN2+VPo6Oji7bFh4eTlFREQUFBdx1111Mnz6dlStXMm7cuFMaBNm6dWtEhOjoaMaOHcvChQtP+lzGNGiFh2DaDS65XPpXX5ILBDfBVHUFqfJfxVlAqqr2Aj4CJgOISGfgdCAFl5QuFJHzyk6iegnuOky091wc8Gug2q+7qjpBVdNVNT05udr1cnw3YMAAvvrqKzZu3AjAwYMHWb9+Pd26dSMzM5OsrCwAXn/99eOeZ+/evbRt6/L7Sy+9dMz9ZsyYQUFBAbm5uXz66af069fvmPuWJpOkpCTy8/OZPn36Cbyzo+3YsQNwSfXdd9+lR48ep3Q+YxqkIwfgtetg48cw/O/Q73bfQglmgskG2pV7nAJsL7+DquaqamlD+0Sgr3f/SmCBquaraj4wFxhQ6dgCYCau2a0TkAYsF5Es77WWiEirGn1HPkhOTuall15i1KhR9OrViwEDBrBu3TpiY2N57rnnGDJkCOeeey4tW7akSZMmxzzPgw8+yMMPP8ygQYMoLi4+5n79+/fn0ksvZcCAAfz2t7+lTZs2x9y3adOmjBs3jp49e3LFFVdUSEblr8F89913pKSk8OSTT/Loo4+SkpLCvn2upXPYsGFl13luvPFGevbsSc+ePdm1axe/+c1vTujfypgG7/B+ePUayPoCrvgH9LnF13DkRJtaAj6xSASwHvgxrpfYIuAGVV1dbp/WqrrDu38l8EtVHSAi1wHjgCG4Suh94Clcs1kjVd3hnf9V4AtVfabSa2cB6dX1IktPT9fKK1quXbuW008//eTfeC3Kz88nISEBVeXuu++mS5cuPPDAAyd9vkceeYSEhAR+/vOf12CUNa8u/Y6MqTUFe11yyc6AqyZAz+B19xeRxaqaXt1+QatgVLUIuAf4AFgLvKGqq0VkvHe9BOBeEVktIsuBe4Ex3vbpwCZgJbAcWK6qs4B4YKaIrPC27wROvKtSPTFx4kR69+7NGWecwd69e7nzzjv9DskY44dDu2HKFbBtMVzzYlCTy4kIWgVTF9T1CqYqL774In/7298qbBs0aBDPPvusTxHVvLr+OzKmRh3IhZevgJx1cO0U6Do06C8ZaAVjI/nrmbFjxzJ27Fi/wzDG1Ib8HJgyAnI3wvVToctFfkdUgSUYY4ypi/Z/B5OHw54tcOMb0PECvyM6iiUYY4ypa/Zug8mXuyRz03RIPdfviKpkCcYYY+qSPVtccjmYBze/A+3P9juiY7IEY4wxdUVepmsWO7wPbn4XUvpWf4yP6u4c8KZG1MZ6MK+++iq9evWiV69eDBw4kOXLjzubjzGmKrs2wIuXwpF8uGVmyCcXsAom5NWH9WDS0tL47LPPaNasGXPnzuWOO+7g66+/9jssY+qOnetgynC3rsuYOdDSv4l4T4QlmOOZ+xB8t7Jmz9mqJwx9/Li71Lf1YAYOHFh2f8CAAWRnZ5/Mv5wxDdN3q1xX5LBwl1xadPM7ooDV3a/F9Vx9XQ9m0qRJDB0a/IFgxtQL25fB5MsgPArGvFenkgtYBXN81VQawVQf14OZN28ekyZN4ssvvzzVfx5j6r/sxfDKlRDdGEbPgsQ0vyM6YZZgQlTl9WCmTp1a4fmlS5ee0PlK14N55513yMrK4oILLqhyv5NZDyYjI4N27drxyCOPHHM9mBUrVnD77bczd+5cmjdvfkKxG9PgbFkAr4yE+OYuuTRt73dEJ8WayEJcfVgPZsuWLVx11VW8/PLLnHbaaceN05gGL+tLePkqaNTSNYvV0eQClmBCXn1YD2b8+PHk5uZy11130bt3b9LTq50jz5iGadM8V7k0becu6DepahHgusNmU67DsynbejDG1CMbPoRpN0LzznDLDEgI3RV3fV8PxgSfrQdjTD2x7j2YdgMkd4Uxs0M6uZwIq2DqcAVTFVsPxpg6Zs0MmH4rtD4TbnoLYpv5HVG1bD2YU6CqR/Wmqivq+3owDfkLkamHVk6Ht++AlHS4cTrENPY7ohplTWSVxMTEkJuba3/IQpCqkpubS0xMjN+hGHPqlr0Gb4+D9ufATW/Xu+QCVsEcJSUlhezs7GpHxht/xMTElA34NKbOWjwZZt0HHc93K1FGxfkdUVBYgqkkMjKybHS8McbUuIUT4b2fQ+eL4LpXIDLW74iCxprIjDGmtsx/1iWXrsPg+tfqdXIBSzDGGFM7vvw/+OBXcPpwuGYyRERXf0wdZ01kxhgTbJ89AfP+AD1GwpX/hPCG8ae3YbxLY4zxgyp88ih88Rc4cxSMeNat69JAWIIxxphgUIUPfwv/+Tv0uQUu+xvU4ZVpT4YlGGOMqWmq8P5D8PXz0O92GPrnBpdcwBKMMcbUrJISeO9/IONfMOBuuOQPUEdnBjlVlmCMMaamlBTDrHth6Stw7gPw4/9tsMkFLMEYY0zNKC6CGXfBitfh/F/CBQ836OQClmCMMebUFRe6ecVWvwMX/gbO+4XfEYUESzDGGHMqio7A9LGwbjZc/HsYdK/fEYUMSzDGGHOyCgvgzdGw/n0Y8icY8FO/IwoplmCMMeZkFB5yq1Bu+gQufRL63eZ3RCEnqB2zRWSIiHwjIhtF5KEqnh8jIjkissy73V7uuSdEZLWIrBWRp8VbAUxE3heR5d5zz4tIuLf9zyKyTkRWiMg7ItI0mO/NGNOAHTkAr14Dm+bB8GcsuRxD0BKM94f/WWAo0B0YJSLdq9j1dVXt7d1e8I4dCAwCegE9gH7A+d7+16rqmd72ZOAab/uHQA9V7QWsBx4OzjszxjRoh/fDKyPh26/cvGJ9bvY7opAVzAqmP7BRVTNV9QgwDRgR4LEKxABRQDQQCXwPoKr7vH0ivOfV2/5vVS3ynlsA2KpUxpiaVbAXXr4Ktn4NV78AZ17nd0QhLZgJpi2wtdzjbG9bZVd7zVrTRaQdgKrOB+YBO7zbB6q6tvQAEfkA2AnsB6ZXcc5bgbk18i6MMQbgYB5MGQHbl8K1k6HH1X5HFPKCmWCqGmFUeaH7WUCq16z1ETAZQEQ6A6fjqpC2wIUicl7ZSVQvAVrjqpsLK7yoyK+BIuDVKoMSuUNEMkQkw5ZFNsYE5EAuTBkO3692q1CefrnfEdUJwUww2UC7co9TgO3ld1DVXFU97D2cCPT17l8JLFDVfFXNx1UjAyodWwDMpFyzm4iMBi4DblTVysms9LgJqpququnJyckn/eaMMQ1E/k546VLYtQFGTYWuQ/yOqM4IZoJZBHQRkTQRiQKuxyWEMiLSutzD4UBpM9gW4HwRiRCRSNwF/rUiklB6jIhEAMOAdd7jIcAvgeGqejCI78sY01Ds2+GSy55v4YY3oPNFfkdUpwRtHIyqFonIPcAHQDjwL1VdLSLjgQxVnQncKyLDcU1aecAY7/DpuKavlbhmtfdVdZaItARmiki0d85PgOe9Y57BNZl96PVoXqCqNurJGHNy9mbD5MtdBXPTW9BhoN8R1TlyjJakBiE9PV0zMjL8DsMYE2p2f+uSy6HdLrm06+93RCFFRBaranp1+9lIfmOMKS93E0weDkf2wy0zoG0fvyOqsyzBGGNMqZz1rrdY0WEYPRta9/I7ojrNEowxxgDsXOsqFxTGzIGWVU08Yk5Ew1sk2hhjKvtupestJmEw5j1LLjXEEowxpmHbvhReugwiYmDse5B8mt8R1RuWYIwxDdfWRTB5BEQ3dsmleSe/I6pXLMEYYxqmb+fDy1dCXKJLLs1S/Y6o3rEEY4xpeDZ/Aa9cDY1aueTStF31x5gTZgnGGNOwbPrELRbWtJ3rLda4jd8R1VuWYIwxDcf6f8Nr17trLWPmQKOWfkdUr1mCMcY0DGtnw7QboEU3GD0L4pP8jqjeswRjjKn/Vr8Db46G1mfCLTPdhX0TdJZgjDH124o3YfqtkNIPbn4HYpv6HVGDYQnGGFN/LX0V3h4HHQbBjdMhprHfETUolmCMMfVTxosw4y7oeIFbLCw6we+IGhxLMMaY+ufrCTD7fujyExg1DaLi/I6oQbIEY4ypX/7zDMz9BXS9FK57BSJj/I6owbLp+o0x9ccXf4WPx0P3K+DqFyA80u+IGjRLMMaYuk8VPvsTfPoY9LwWrvgHhNufN7/Zb8AYU7epuqrlyyeh940w/O8QFu53VAZLMMaYukwV/v0bmP8M9B0Dl/4fhNml5VBhCcYYUzepwtwHYeEE6H8HDH0CRPyOypRjCcYYU/eUlMCcB2DxS3DOPfCTRy25hKATqiVFJExEbCisMcY/JcUw8x6XXM79mSWXEFZtghGR10SksYjEA2uAb0TkF8EPzRhjKikugnfuhGWvwgUPw49/Z8klhAVSwXRX1X3AFcB7QHvg5qBGZYwxlRUXwlu3wco3XWK54CFLLiEukAQTKSKRuAQzQ1ULAQ1uWMYYU07RYXhzDKx5F37yB/jR//gdkQlAIAnmn0AWEA98LiIdgH3BDMoYY8oUFsDrN8G62TD0zzDwHr8jMgGqtheZqj4NPF1u07ciMjh4IRljjOfIQbcKZeY8uOwpSB/rd0TmBARykf8+7yK/iMgkEVkCXFgLsRljGrLD+fDatZD5KYx41pJLHRRIE9mt3kX+nwDJwFjg8aBGZYxp2Ar2wStXw7dfwVUT4Kyb/I7InIRABlqWdtMYBryoqstFrOuGMSZIDu1xyWXHMhj5LzjjSr8jMicpkASzWET+DaQBD4tII6AkuGEZYxqkg3nw8pXw/Wq4dgp0u9TviMwpCCTB3Ab0BjJV9aCINMc1kxljTM05sAumjIBdG+D61+C0n/gdkTlF1V6DUdUSIAX4jYj8BRioqisCObmIDBGRb0Rko4g8VMXzY0QkR0SWebfbyz33hIisFpG1IvJ0abOciLwvIsu9554XkXBve6KIfCgiG7yfzQL8NzDG+G3/9/DSpZC7EUZNteRSTwTSi+xx4D7cNDFrgHtF5LEAjgsHngWGAt2BUSLSvYpdX1fV3t7tBe/YgcAgoBfQA+gHnO/tf62qnultTwau8bY/BHysql2Aj73HxphQt287vDQM9myBG9+Ezj/2OyJTQwJpIhsG9PYqGURkMrAUeLia4/oDG1U10ztuGjACl6Sqo0AMEIXrZBAJfA/g9WgrjT2KH2YVGAFc4N2fDHwK/DKA1zLG+GXPVph8uWseu+lt6HCO3xGZGhTobMpNy91vEuAxbYGt5R5ne9squ1pEVojIdBFpB6Cq84F5wA7v9oGqri09QEQ+AHYC+4Hp3uaWqrrDO34H0CLAOI0xftid5SqXg3lw8zuWXOqhQBLMY8BSEXnJq14WA38M4LiqujJXnsNsFpCqqr2Aj3CVByLSGTgdd+2nLXChiJxXdhLVS4DWQDQnOOhTRO4QkQwRycjJyTmRQ40xNSV3E7w4zI13GT0D2vXzOyITBIFc5J8KDADe9m7nAJ8HcO5soF25xynA9krnzlXVw97DiUBf7/6VwAJVzVfVfGCuF0P5YwuAmbimMYDvRaQ1gPdz5zHezwRVTVfV9OTk5ADehjGmRuWsd8mlqADGzIY2Z/kdkQmSgJrIVHWHqs5U1Rmq+h2wIIDDFgFdRCRNRKKA63EJoUxpQvAMB0qbwbYA54tIhDeT8/nAWhFJKJdEInDXh9Z5x8wERnv3RwMzAnlvxpha9P0a1yymJTBmDrTq6XdEJohOdsnkakfyq2qRiNwDfACEA/9S1dUiMh7IUNWZuB5pw4EiIA8Y4x0+Hdf0tRLXrPa+qs4SkZbATBGJ9s75CfC8d8zjwBsichsuQZX2LjPGhIIdK9w4l4hoGD0Lkrr4HZEJMlE98aVdRGSLqrYPQjy1Kj09XTMyMvwOw5j6b9sSN0I/KgFGz4TmnfyOyJwCEVmsqunV7XfMCkZE/k7VC4sJFXuVGWPMsW1d6OYWi20Ko2dDsw5+R2RqyfGayI731d6+9htjqvftf+DVayChhWsWa5Lid0SmFh0zwajq5NoMxBhTz2R+BlOvd0nllpnQuHX1x5h6JdCBlsYYE7iNH7nFwpqlut5illwaJEswxpia9c37MHWU6yU2erZrHjMNkiUYY0zNWTsLXr8JWp7hmsXim/sdkfFRteNgROTpKjbvxY1lscGMxhhn1Vvw1jho2wduegtiAp220NRXgVQwMbgFxzZ4t15AInCbiDwVxNiMMXXF8tfhrduh3dlu4kpLLobARvJ3Bi5U1SIAEfkH8G/gYtxIe2NMQ7bkZZj535B6LtzwOkTF+x2RCRGBVDBtgfL/Y+KBNqpaDByu+hBjTIOwaBLMvAc6DYYb3rDkYioIpIJ5AlgmIp/iRvGfB/xRROJxU+wbYxqiBc/D+7+ELpfAtVMgMsbviEyIqTbBqOokEXkPt0KlAL9S1dJp938RzOCMMSHqq6fhw99Ct8tg5IsQEeV3RCYEBTqbcj/gR979Yiqt62KMaUA+/zN88iiccRVcNQHCI/2OyISoaq/BiMjjwH3AGu92r4g8FuzAjDEhRhXm/dEll17XwVUTLbmY4wqkghkG9FbVEgBv2eSlwMPBDMwYE0JU4aNH4KunoPdNMPxpCAv3OyoT4gIdyV9+en7r4G5MQ6IKH/zaJZf0W2H43y25mIAEUsE8BiwVkXn80IvMqhdjGoKSEpj7ICyaCGf/FIY8DlLtgrbGAIH1IpvqdVHuh0swv8TmMDOm/ispgdn3wZIpMPBeuHi8JRdzQgLqRaaqO4CZpY9FZAtQ55dMNsYcQ0kxzLgHlr8GP/o5XPgbSy7mhAXaTbky+59mTH1VXATv3AmrpsPgX8P5D/odkamjTjbBaI1GYYwJDcWFMP1WWDsTLnoEzn3A74hMHXbMBCMif6fqRCJU7FVmjKkPig7Dm2Pgm/fgkj/COXf7HZGp445XwWSc5HPGmLqm8BC8fjNs/BCG/QX6j/M7IlMPHDPBqOrkyttEpJWqfhfckIwxterIQZg2CjI/g8v/Bn3H+B2RqSdOtLvxe0GJwhjjj8P58Oo1sPlzuOI5Sy6mRp3oRX7rPWZMfVGwD14dCdkZbl6xniP9jsjUMyeaYCYGJQpjTO06tBteuRp2LIdrXoTuI/yOyNRDJ5RgVPW5YAVizMnafeAIi7LyWLp1D/FR4XRKTqBjcgKpSXFER9icWUc5mAdTRsDOtXDty9BtmN8RmXrqZMfBGOOb7/YW8PXmXBZl5bFwcx7rv88HICJMKCr5oWd9mEC7xDg6JsWXJZ1OyfF0TE4gKSEKaYgj0/NzXHLJ3QijpkKXi/2OyNRjlmBMSFNVsnIPsnBzLgs372ZhVi5b8w4BkBAdQZ8OzRjRuy39UhPpldKE4hJl864DbMrJZ1POATK9n/MzcykoLCk7b+OYCDomJ9Ax2SWfTt7P9s3rcdWz/zuYPBz2bIEbXodOg/2OyNRzlmBMSCkpUdZ9t7+sOlmYlUfO/sMAJMZH0S+1GWMGptE/NZHTWzciIvzojpA92jahR9smR513+95DZOa45FP68z8bc3l7ybay/cIE2ifGVah2OnmJqHl8Ha569m2HyZfDvh1w03RIPdfviEwDYAnG+OpIUQkrt+0tSygZWXnsKygCoE2TGAZ1ak7/tOb0T2tGp+SEk/4DHxYmpDSLI6VZHOedllzhufzDRWwuSzyu4tmUk89XG3dxuOiHqqdJbGRZxVO+8mmfGE9URAhPML5ni0suB3Lh5reh/QC/IzINhCUYU6sOHili6ZY9rjrZnMfSrbvLmq46Jcdzaa/W9E9LpF9qIinN4molpoToCHqmNKFnytFVz7Y9hypUPJk5B/h8fQ7TF2eX7RceJrRPjCtX8fxQ+STGR9XKezimvM0uuRTsg1tmQEpff+MxDYolGBNUew8WkvGtSyZfb85j1ba9FJUoYQLd2zRmVP/2nJ2WSHpqIkkJ0X6HW0FYmNAuMY52iXFc0LXic/sLCsnMOUDmrnw27fzh5+cbdnGkXNXTNC7SVTxJ8XRq8cPP9olxRFbRvFejdm10yaXoEIyeCW16B/f1jKlEVIM3MbKIDAH+BoQDL6jq45WeHwP8GShtBH9GVV/wnnsCuBQ328CHwH1ALPAm0AkoBmap6kPe/u2BybiJOMOBh1T1uDMPpKena0aGTatWk3buK2Bh6fWTzXl88/1+VCEqPIxeKU1cdZKWSN8OzWgcE+l3uDWuuETZtvsQm3bls2lnPpm7DpT9LL2WBK7HW/vmcXRMSqBTi3g6eT87JiXQrCaqnpxvXHIpKXaVS6sep35OYzwislhV06vbL2gVjIiEA88CFwPZwCIRmamqayrt+rqq3lPp2IHAIKCXt+lL4HxgIfAXVZ0nIlHAxyIyVFXnAr8B3lDVf4hId9y0NqlBensG18NrS97BsmSyKCuPrNyDAMRFhdO3QzMu7dmafmmJ9G7XlJjIeto7q5xwL3G0bx7H4K4tKjy3z6t6XML5ofL5fH0OR4p/qHoS46PKda3+4Wf7xLgqOzUc5fvVrrdYWDiMmQMtutX02zQmIMFsIusPbFTVTAARmQaMAConmKooEANE4aaniQS+V9WDwDwAVT0iIkuAlHLHNPbuNwG219D7MJ6SEmXDznwWbs7lay+hfL/PfStvGhdJv9REbhrQgX6piZzRpnFgfwwbkMYxkfRu15Te7SqudlFcomTvPljhWs+mnAN8vO57Xs84UrZfZHjptZ6KY3o6JyfQJM6rBncshylXQEQMjJ4FSZ1r8y0aU0EwE0xbYGu5x9nA2VXsd7WInAesBx5Q1a2qOl9E5gE7cAnmGVVdW/4gEWkKXI5rggN4BPi3iPw3EA9cVFVQInIHcAdA+/a26vPxFBaXsHr7vrIxKIuy8th7qBCAVo1jODutOf3SEjk7LZHOyQmEhdXRLrw+Cw8TOjSPp0PzeC6sVGzsPVjIpl3lOxm45DPvm50UFv/QvN08PoqfNNnG7/b8mqLIBFaeN4U2tCaluMQSvfFNMBNMVX9tKl/wmQVMVdXDIvJT3DWUC0WkM3A6P1QnH4rIear6OYCIRABTgadLKyRgFPCSqv5VRM4BXhaRHqpaUv4FVXUCMAHcNZhTf5v1R0FhcVkPr0VZeSz+djeHCosBSEuKZ8gZrcoSSkqz2Lo7JqQOaRIXSZ/2zejTvlmF7UXFJWzdfcj8mrc4AAAaM0lEQVRLOPkUZS1gzOaHyNVGXLf3Yba9tRPYSWS4kNo8vlxT2w+VT5PY+ncNzISWYCaYbKBduccpVGq2UtXccg8nAn/y7l8JLFDVfAARmQsMAD73np8AbFDVp8odfxswxDvvfBGJAZKAnTXybuqhfQWFLM7aXdbctSJ7D4XFigh0a9WY6/q1o19qIv3SmtGiUYzf4ZpyIsLDSEuKJy0pnh/HboQvfgHN2hA3ehZzIpPLxvKUVj4bd+bz8dqdFabSSUqIPmomg47J8aQ0iyPcqlFTA4KZYBYBXUQkDddL7HrghvI7iEhrVd3hPRwOlDaDbQHGichjuErofOAp75hHcddYbq/0eluAHwMvicjpuGs4OTX9puqynP2HfxghvzmPtd/tQ9X1aOqV0oRbz03j7LRE+nZItG+3dUXmp/Da9dC0veuK3KgVTYG+HaLo26Fi1VNYXMLWvIPlptBxCej9VTvYfbCwbL+oiDBSm8dV6mTg7tfHnn8meILdTXkYLjGEA/9S1T+IyHggQ1VneglkOFAE5AH/parrvB5ozwHn4ZrV3lfVn4lICu66zjqgtM/nM6r6gtdzbCKQ4B3zoKr++3jx1eduyqpK9u5DZc1dCzfnkbnrAACxkeH06dCUfqmJ9E9L5Kx2zYiNqv89vOqdDR/B6zdCYifXFTkhufpjjiHvwBEyK3UyyMzJ59u8gxSXq3qSG0UfNYVO5+QE2jSNtaqnAQm0m3JQE0yoq08JRlXZuDO/rLlr4eY8duwtANzEjqWj4/unJdKjbZPgD/IzwfXNXHjjFkjuCjfPgPjmQXmZI0UlbMk7WNa5oLTy2ZRzoKzDB7iqp2NS/FFT6XRMTiAh2sZz1ze+j4MxwVVUXMKaHfsqjEEpbeZo0Sia/mmJZbfTWjSyHl71yZoZMP1WaNXLzS0W26z6Y05SVEQYnVsk0LlFQoXtquqqnnIDSTNz8lm7Yz8frP6+QtXTsnF02YBS99PNaNC2aaz9v6znLMHUEQWFxSzfuodFWW7KlSXf7ubAEdfDq0PzOH58ekuXUFIT6dA8znp41Vcrp8Pbd0BKOtz4JsQ0qf6YIBARmidE0zwhmn6piRWec1XPgaM6Gsxctr1sIlOAmMgwUpu7qXM6lU2l46qfeKt66gX7LYao/QWFLP52d1lz1/Kte8tGe3dr1Yir+qSUVSgtG1sPrwZh2VSYcRe0P8et5xLdyO+IquSqnkZ0blExPlUl98CRo8b0rN62l7krd1Cu6KFV45gfKp7Saz4tEmjdOMaqnjrEEkyIyM0/zKKs3d4aKLms2b6PEnWD8Hq2bcKYQan0T00kPbUZTeN8nqHX1L4lU2DmvZB2nluJMire74hOmIiQlBBNUoJrwi3vcFExW3IPll3fKf357rJt7K9U9ZRWOZWn0omLsj9nocZ+Iz7ZtucQi7wZhhdl5bFxp1v2NzoijLPaN+WeC7vQPzWRs9o3teaChm7hRHjv59D5IrjuFYiM9TuiGhcdEU6Xlo3o0vLoqicn//BRC8WtyN7LnJU7KN9HqU2TmCoXimvdJMaajH1if7lqgaqyKedAhTEo2/a4ZX8bxUSQ3qEZV/dJoX9aM3q2bRrai1eZ2jX/OfjgYThtKFw7GSJCa0mDYBMRWjSKoUWjGAZ0rNhTrqCwmG9zD5Y1t5Umn7eXbGP/4R+qnriocNKqmDy0Y1KCdc8PMkswQVBcoqyt1MMr94CbtNA1DzRj3I/S6JeWSLdWjW38gKnal0/BR/8Lpw+HqydBhDWNlhcTGU7XVo3o2qqKqmf/4aM6GSzduptZK7ZXqHraNo2tMJtBaeXTsnG0VT01wBJMDThcVMzK7L1lzV2Ls3aXfYNqlxjL+V2TOdsbh5KWFG//cU31PnsC5v0BelwNV06AcPuoBkpEaNE4hhaNYzin09FVT1buAbdUQulsBrsO8GbG1rJemQDxUeFlsxeUr3zSkuIbxLITNcX+156EA4eLWLJld1mFsmzrnrK1209rmcDw3m3Keni1blL/2stNEKm6xPL5n6HX9XDFc25dF1MjYiLD6daqMd1aNa6wXVXZuf8wm3bms6nc2J6MrN3MWPbDFIoipVVPQoUlsjslJ9CikVU9lVmCOQkTv8jkqY82EB4mnNGmMTcP6EA/r0LxfQ12U3epwoe/g/88DWfdDJf/zZJLLRERWjaOoWXjGAZ2Tqrw3KEjxWzeVWl57Jx8MrLyOFiu6kmIjvih4ikd15McT2rzhlv12FQxJzFVTNauA2zJO0ifDs1sGgxTM1Th/Yfh639A+m0w7C8QZp09Qpmq8t2+gqN6uGXmHCjrxAOu6klpFuslnvIzGsSTnFA3qx6bKiaIUpPiSU2qe+MQTIgqKXHdkDMmwYC74JI/ur9KJqSJCK2bxNK6SSyDKlU9B48UsXnXgXLzt7mfX2fmla2xBNAoOoKOFWYycD87NI8jOqLuVz2WYIzxU0kxzLoPlr4Mg+6Di/6fJZd6IC4qgjPaNOGMNhWn8ikpcVVP5YpnfmYuby/dVrZfmEC7xDiXcCotFJeUEFVnqh5LMMb4pbgIZtwNK6bBeQ/C4F9ZcqnnwsKENk1jadM0lh91qbi8woHDpVVPfoXKZ35mLgWFPyzM2zgmosJA0tIu1h2ax4fcGDpLMCfjYB4c2g3NO/kdiamrigvdpJWr34bBv4Hzf+F3RMZn8dER9GjbhB5tj656tu89dNS1nq827uKtJdll+4WHCe1Kr/VUWiI7Md6fqscSzMnImASfPAqdL4b+49wUHtbbxwSq6Ai8dSusnQUXj3dNY8YcQ1iYkNIsjpRmcZx3WsWqJ/9wEZsrTR66KSefLzfuKhs6AdAkNvKoKXTOateUFkGeKNd6kZ3MgmP7dsCSyZDxIuR/B007QL/bXNfSuMTqjzcNV9FheGM0rJ8LQx6HAf/ld0SmHiopUbbtOXTUtZ5NOfns3O8WA/79FT24eUCHkzq/rWgZgFNe0bK4ENbNhoUvwLdfQng09BwJ/W6Htn1qLlBTPxQegmk3wqaP4dK/uv8nxtSy/QWFZOYcoHWTmJOuYCzBBKBGl0z+fg0segGWT4PCA9C2r/sDcsZVEGnrtTR4Rw7A1Oth8xcw/Gnoc4vfERlz0izBBKBGE0ypgn0uySyaCLvWQ2yi+2OSfis0O7ly1NRxh/fDq9fC1gVwxT/gzOv9jsiYU2IJJgBBSTClVGHz57BwAnzznnt82hDofzt0vNBGaTcUBXvhlZGwbTFcPdFNXmlMHWcj+f0mAh3Pd7e92a5DwJLJ8MpcSOzoms963wCxzfyO1ATLod3w8lXw3Qq45iXoPtzviIypVVbBBKuCqUrREVg701U1W7+GiFjodQ30Gwete9VeHCb4DuTCyyMg5xu4dgp0Hep3RMbUGKtgQlFElOtl1nMk7FjhrtOseNOtt97ubJdoug9vcKsW1jv5O2HKCMjLhOunQpeL/I7IGF9YBVObFUxVDu2GZa+5Hmh5mRCfDH1GQ/pYaJLib2zmxO3/DiZfDnu2wg3ToOMFfkdkTI2zi/wBCIkEU6qkBDI/cWNq1r/vruF0HeZmCkg73+aoqgv2bnPJJf97uOENSB3kd0TGBIU1kdU1YWFuypnOF8HubyHjX67pbN1sSDrNdQo483qIaVL9uUzt2/2tSy6HdsNNb0P7s/2OyBjfWQUTKhVMVQoLYPU77lrNtsUQGQ9nXueu1bTs7nd0plReJkweDof3wc3vuEG2xtRj1kQWgJBPMOVtW+Ku06ycDsWHocMgV9WcfjmER/odXcO1a4OrXIoOwy3vQusz/Y7ImKCzBBOAOpVgSh3Mc4tTLZoEe76FhFbQd4y7NW7td3QNy851LrmgcMsMaHmG3xEZUysswQSgTiaYUiXFsPEjWDjR/QwLh26XuU4BHQZZp4Bg+26V64ocFgGjZ0JyV78jMqbW2EX++i4sHE67xN3yMl1Fs/QVWPMutOjulg/odR1EN/I70vpn+zJ4+QqIjIPRs2zhOWOOwSqYulrBVOXIQVj1lusUsGM5RDWC3qPctRr7hl0zsjPc9C8xTVzlkpjmd0TG1LpAK5igzrgoIkNE5BsR2SgiD1Xx/BgRyRGRZd7t9nLPPSEiq0VkrYg8LU6ciMwRkXXec49XOt+1IrLGe+61YL63kBQVB31uhjs+g9s+gm7DYPFL8Gx/d61gzUy3Drw5Od/OhylXQFwzGDvHkosx1QhaE5mIhAPPAhcD2cAiEZmpqmsq7fq6qt5T6diBwCCgdIKuL4HzgYXAX1R1nohEAR+LyFBVnSsiXYCHgUGqultEWgTrvYU8EWjXz91+8gdYOsVNtvnGzdC4LfQdC31HQ0LD/Sc6YZu/gNeucx0pRs+Cxm38jsiYkBfMCqY/sFFVM1X1CDANGBHgsQrEAFFANBAJfK+qB1V1HoB3ziVA6Xwq44BnVXW39/zOGnsndVlCMvzof+DeZXD9a5DUBeY9Ck92h+m3wZYFbikBc2yb5sGr10DTdjDmPUsuxgQomAmmLbC13ONsb1tlV4vIChGZLiLtAFR1PjAP2OHdPlDVteUPEpGmwOXAx96m04DTROQrEVkgIkNq9u3UceER0O1S1532ngx3XWbDh/CvS+CfP3JNaUcO+B1l6Fn/b1e5JHaE0bOhUUu/IzKmzghmgqmqn2zlr8qzgFRV7QV8BEwGEJHOwOm46qQtcKGInFd2YpEIYCrwtKpmepsjgC7ABcAo4AUvCVUMSuQOEckQkYycnJxTeHt1WFIXGPo4/M9auOwpNw/arPvgr6fD+w9D7ia/IwwN6+bAtBugRTcYM9tVg8aYgAUzwWQD7co9TgG2l99BVXNV9bD3cCJQOsfGlcACVc1X1XxgLjCg3KETgA2q+lSl15uhqoWquhn4BpdwKlDVCaqarqrpyckN/A9GVLybtfm/voKxc6Hzj91aNX/vAy9fCevec+NtGqLV78Ibt7h1em6ZCXGJfkdkTJ0TzASzCOgiImneBfnrgZnldxCR8kPPhwOlzWBbgPNFJEJEInEX+Nd6xzwKNAHur/R67wKDvX2ScE1mmZjqiUCHgXDNi/DAarjgV7BzLUwbBX/rDV88CQd2+R1l7VnxJky/Fdqmw83vQuxRhbAxJgBBSzCqWgTcA3yASw5vqOpqERkvIqVrx97rdSleDtwLjPG2Twc2ASuB5cByVZ0lIinAr4HuwJJKXZs/AHJFZA3u+s0vVDU3WO+v3mrUCi74Jdy/Eq6ZDM06wMf/z3UKePtOyF7sd4TBtew1eHsctD8HbnoLYhr7HZExdZYNtKxPAy2DZedaN9Hm8mlwJB/anOU6CfS4GiJj/Y6u5ix+CWbdDx3PdytRRsX5HZExIcnmIguAJZgTVLAPVrzu5j/b9Q3ENoOzboL02+r+oMOFE+G9n0Pni+G6VyAyxu+IjAlZlmACYAnmJKlC1hfuj/K6OaAl0OVit05N54vc4ml1yfxn4YNfQddL3XWoiGi/IzImpNlklyZ4RCDtPHfbu801LS1+CV67BpqluormrJvqRs+rL55015i6j4CrJ9naOsbUIKtgrIKpGUVHYO1Md61my3yIiIGeI11V06a339EdTRU+ewI+/SP0vAaueN4NRjXGVMsqGFO7IqJcQuk5Er5b6RLNijfcEgIp/VyiOeOK0Gh+UoVPfg9f/BXOvAFGPOOWPzDG1CirYKyCCZ5De2D5VJdscjdCXBL0uQXSb3XzevlBFf79G5j/DPQZ7WYyqGvXjIzxmV3kD4AlmFpSUgKbP4WFL8D6uW7baUOh/+2QdkHt/YFXhbm/hIX/dBXV0CcsuRhzEqyJzISOsDDodKG77dnilg5YMhm+mQPNO7sxNWeOCu6I+ZISmPMzWPwiDLgbLvmDLSttTJBZBWMVjD+KDrv5vhZNhOxFbvnhXte6yqJVj5p9rZJimHkvLHsFzn0Afvy/llyMOQVWwZjQFhENZ17nbtuXuUSzfJrr7tz+HFfVnD7cdR44FcVF8O5/wco34PyH4IKHLLkYU0usgrEKJnQczINlr7pOAbuzIL4F9B3jZnw+mUW+igvdvGKr34ELfwvn/bymIzamQbKL/AGwBBOiSkpg08dupoAN/wYJc4ul9R8HqT8KrAIpOgLTx8K62fCTR2Hgfwc/bmMaCGsiM3VXWJibeqbLxZC3GTL+BUtfdgM5k7u55rNe1x17puPCAreWy4YPXE+xs++s3fiNMYBVMFbB1BWFh2DVW66q2bEMohLgzOtdsmlx+g/7HTkIr98Imz6By/7PjbkxxtQoayILgCWYOkgVti12iWb121B8xDWb9bsdOl4Ar98EWV+60fln3eR3tMbUS5ZgAmAJpo47sAuWTHHjavZugbBI0GI3r9iZ1/kdnTH1ll2DMfVffBL86Gcw6D5Y/4GblqbnNdB9ePXHGmOCzhKMqfvCwqHbMHczxoQMm4jJGGNMUFiCMcYYExSWYIwxxgSFJRhjjDFBYQnGGGNMUFiCMcYYExSWYIwxxgSFJRhjjDFB0aCnihGRHOBbv+MoJwnY5XcQxxHq8UHoxxjq8UHoxxjq8UH9j7GDqiZXt1ODTjChRkQyApnfxy+hHh+EfoyhHh+EfoyhHh9YjKWsicwYY0xQWIIxxhgTFJZgQssEvwOoRqjHB6EfY6jHB6EfY6jHBxYjYNdgjDHGBIlVMMYYY4LCEkwtE5EhIvKNiGwUkYeOsc+1IrJGRFaLyGuhFqOItBeReSKyVERWiEitLsQiIv8SkZ0isuoYz4uIPO3Fv0JE+oRYfDd6ca0Qkf+IyJm1GV8gMZbbr5+IFIvIyNqKrdxrVxujiFwgIsu8z8pnoRSfiDQRkVkistyLb2wtx9fO+5yu9V7/vir2Ce5nRVXtVks3IBzYBHQEooDlQPdK+3QBlgLNvMctQjDGCcB/efe7A1m1HON5QB9g1TGeHwbMBQQYAHwdYvENLPf7HVrb8QUSY7n/C58A7wEjQy1GoCmwBmjvPa7tz0p18f0K+JN3PxnIA6JqMb7WQB/vfiNgfRWf5aB+VqyCqV39gY2qmqmqR4BpwIhK+4wDnlXV3QCqujMEY1SgsXe/CbC9FuNDVT/HfViPZQQwRZ0FQFMRaV070VUfn6r+p/T3CywAUmolsIoxVPdvCPDfwFtAbf8fBAKK8QbgbVXd4u1fq3EGEJ8CjUREgARv36LaiA1AVXeo6hLv/n5gLdC20m5B/axYgqldbYGt5R5nc/Qv/DTgNBH5SkQWiMiQWovOCSTGR4CbRCQb9+32v2sntIAF8h5CxW24b5AhRUTaAlcCz/sdy3GcBjQTkU9FZLGI3OJ3QJU8A5yO+wK2ErhPVUv8CEREUoGzgK8rPRXUz0pETZ3IBESq2Fa5G18ErpnsAtw32y9EpIeq7glybKUCiXEU8JKq/lVEzgFe9mL05cNThUDeg+9EZDAuwZzrdyxVeAr4paoWuy/gISkC6Av8GIgF5ovIAlVd729YZS4BlgEXAp2AD0XkC1XdV5tBiEgCrhK9v4rXDupnxRJM7coG2pV7nMLRzUvZwAJVLQQ2i8g3uISzqHZCDCjG24AhAKo6X0RicPMa+dKUUoVA3oOvRKQX8AIwVFVz/Y6nCunANC+5JAHDRKRIVd/1N6wKsoFdqnoAOCAinwNn4q41hIKxwOPqLnZsFJHNQDdgYW0FICKRuOTyqqq+XcUuQf2sWBNZ7VoEdBGRNBGJAq4HZlba511gMICIJOGaATJDLMYtuG+NiMjpQAyQU4sxVmcmcIvXQ2YAsFdVd/gdVCkRaQ+8DdwcQt+2K1DVNFVNVdVUYDpwV4glF4AZwI9EJEJE4oCzcdcZQkX5z0lLoCu1+Fn2rv1MAtaq6pPH2C2onxWrYGqRqhaJyD3AB7geOv9S1dUiMh7IUNWZ3nM/EZE1QDHwi9r8hhtgjP8DTBSRB3Dl9BjvW1qtEJGpuCbEJO860P8CkV78z+OuCw0DNgIHcd8ka00A8f0OaA4851UIRVrLEyMGEKPvqotRVdeKyPvACqAEeEFVj9vtujbjA34PvCQiK3FNUb9U1dqcYXkQcDOwUkSWedt+BbQvF2NQPys2kt8YY0xQWBOZMcaYoLAEY4wxJigswRhjjAkKSzDGGGOCwhKMMcaYoLAEY4wxJigswRhTB3jT0s8+1X2MqU2WYIypAd5IaPs8GVOOfSCMOUkikuot5vQcsAS4WUTmi8gSEXnTm2QQERkmIutE5EtvcadjVhki0l/cImRLvZ9dq9jnERF5WUQ+EZENIjKu3NMJIjLde71XvelCEJHficgiEVklIhNKtxsTTJZgjDk1XYEpwMW4SUAvUtU+QAbwM28i0H/iJrU8F7fw1PGsA85T1bNwU8r88Rj79QIuBc4BficibbztZwH34xaC64ibLgTgGVXtp6o9cDMPX3bC79SYE2RzkRlzar5V1QUichnuj/pXXnEQBczHzZ6bqaqbvf2nAncc53xNgMki0gU3z1vkMfaboaqHgEMiMg+3UNweYKGqZgN480+lAl8Cg0XkQSAOSARWA7NO7i0bExhLMMacmgPeTwE+VNVR5Z8UkbNO8Hy/B+ap6pXeIlGfHmO/ypMIlj4+XG5bMRDhVVHPAemqulVEHsHNgG1MUFkTmTE1YwEwSEQ6A4hInIichmvy6uglC4DrqjlPE2Cbd3/McfYbISIxItIcN6Pv8dYLKk0mu7zrQiOricGYGmEJxpgaoKo5uIQwVURW4BJON68Z6y7gfRH5Evge2HucUz0BPCYiX+GWSziWhcAc73V+r6rHXCTKWw11Im7Z3nepvcXrTANn0/UbE2QikqCq+V7PrWeBDar6f6dwvkeAfFX9S03FaEwwWAVjTPCN8y64r8Y1gf3T53iMqRVWwRjjAxEZC9xXafNXqnq3H/EYEwyWYIwxxgSFNZEZY4wJCkswxhhjgsISjDHGmKCwBGOMMSYoLMEYY4wJiv8PoHI2pRJekUkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10cf8a250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 最终结果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过一系列的调优，得到最优参数:\n",
    "bestparam= {'n_estimators':193, \n",
    "            'max_depth':6, \n",
    "            'min_child_weight':6, \n",
    "            'subsample': 0.7, \n",
    "            'colsample_bytree':0.7, \n",
    "            'reg_alpha':2, \n",
    "            'reg_lambda':1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
